{
 "cells": [
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-10T07:02:51.038631Z",
     "start_time": "2025-03-10T07:02:30.835218Z"
    }
   },
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "from tqdm.auto import tqdm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "print(sys.version_info)\n",
    "for module in mpl, np, pd, sklearn, torch:\n",
    "    print(module.__name__, module.__version__)\n",
    "    \n",
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "print(device)\n",
    "\n",
    "seed = 42\n",
    "torch.manual_seed(seed)\n",
    "torch.cuda.manual_seed_all(seed)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sys.version_info(major=3, minor=12, micro=9, releaselevel='final', serial=0)\n",
      "matplotlib 3.10.1\n",
      "numpy 2.0.1\n",
      "pandas 2.2.3\n",
      "sklearn 1.6.1\n",
      "torch 2.6.0+cu126\n",
      "cuda:0\n"
     ]
    }
   ],
   "execution_count": 1
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据准备"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-10T07:02:51.043849Z",
     "start_time": "2025-03-10T07:02:51.039626Z"
    }
   },
   "source": [
    "# !wget https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt"
   ],
   "outputs": [],
   "execution_count": 2
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-10T07:02:51.091041Z",
     "start_time": "2025-03-10T07:02:51.044845Z"
    }
   },
   "source": [
    "# https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt\n",
    "#文件已经下载好了\n",
    "with open(\"./shakespeare.txt\", \"r\", encoding=\"utf8\") as file:\n",
    "    text = file.read()\n",
    "\n",
    "print(\"length\", len(text))\n",
    "print(text[0:100])"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length 1115394\n",
      "First Citizen:\n",
      "Before we proceed any further, hear me speak.\n",
      "\n",
      "All:\n",
      "Speak, speak.\n",
      "\n",
      "First Citizen:\n",
      "You\n"
     ]
    }
   ],
   "execution_count": 3
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 构造字典"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-10T07:02:51.109011Z",
     "start_time": "2025-03-10T07:02:51.093035Z"
    }
   },
   "source": [
    "# 1. generate vocab\n",
    "# 2. build mapping char->id\n",
    "# 3. data -> id_data  把数据都转为id\n",
    "# 4. a b c d [EOS] -> [BOS] b c d  预测下一个字符生成的模型，也就是输入是a，输出就是b\n",
    "\n",
    "#去重，留下独立字符，并排序（排序是为了好看）\n",
    "vocab = sorted(set(text))# 利用set去重，sorted排序\n",
    "print(len(vocab))\n",
    "print(vocab)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "65\n",
      "['\\n', ' ', '!', '$', '&', \"'\", ',', '-', '.', '3', ':', ';', '?', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']\n"
     ]
    }
   ],
   "execution_count": 4
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-10T07:02:51.114368Z",
     "start_time": "2025-03-10T07:02:51.110005Z"
    }
   },
   "source": [
    "for idx,char in enumerate(['h','o','w']):\n",
    "    print(idx,char)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 h\n",
      "1 o\n",
      "2 w\n"
     ]
    }
   ],
   "execution_count": 5
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-10T07:02:51.120880Z",
     "start_time": "2025-03-10T07:02:51.115364Z"
    }
   },
   "source": [
    "#每个字符都编好号，enumerate对每一个位置编号，生成的是列表中是元组，下面字典生成式\n",
    "char2idx = {char:idx for idx, char in enumerate(vocab)}\n",
    "print(char2idx)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'\\n': 0, ' ': 1, '!': 2, '$': 3, '&': 4, \"'\": 5, ',': 6, '-': 7, '.': 8, '3': 9, ':': 10, ';': 11, '?': 12, 'A': 13, 'B': 14, 'C': 15, 'D': 16, 'E': 17, 'F': 18, 'G': 19, 'H': 20, 'I': 21, 'J': 22, 'K': 23, 'L': 24, 'M': 25, 'N': 26, 'O': 27, 'P': 28, 'Q': 29, 'R': 30, 'S': 31, 'T': 32, 'U': 33, 'V': 34, 'W': 35, 'X': 36, 'Y': 37, 'Z': 38, 'a': 39, 'b': 40, 'c': 41, 'd': 42, 'e': 43, 'f': 44, 'g': 45, 'h': 46, 'i': 47, 'j': 48, 'k': 49, 'l': 50, 'm': 51, 'n': 52, 'o': 53, 'p': 54, 'q': 55, 'r': 56, 's': 57, 't': 58, 'u': 59, 'v': 60, 'w': 61, 'x': 62, 'y': 63, 'z': 64}\n"
     ]
    }
   ],
   "execution_count": 6
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-10T07:02:51.130640Z",
     "start_time": "2025-03-10T07:02:51.122869Z"
    }
   },
   "source": [
    "# 把vocab从列表变为ndarray\n",
    "idx2char = np.array(vocab)\n",
    "print(idx2char)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['\\n' ' ' '!' '$' '&' \"'\" ',' '-' '.' '3' ':' ';' '?' 'A' 'B' 'C' 'D' 'E'\n",
      " 'F' 'G' 'H' 'I' 'J' 'K' 'L' 'M' 'N' 'O' 'P' 'Q' 'R' 'S' 'T' 'U' 'V' 'W'\n",
      " 'X' 'Y' 'Z' 'a' 'b' 'c' 'd' 'e' 'f' 'g' 'h' 'i' 'j' 'k' 'l' 'm' 'n' 'o'\n",
      " 'p' 'q' 'r' 's' 't' 'u' 'v' 'w' 'x' 'y' 'z']\n"
     ]
    }
   ],
   "execution_count": 7
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-10T07:02:51.292615Z",
     "start_time": "2025-03-10T07:02:51.132348Z"
    }
   },
   "source": [
    "#把字符都转换为id\n",
    "text_as_int = np.array([char2idx[c] for c in text])\n",
    "print(text_as_int.shape)\n",
    "print(len(text_as_int))\n",
    "print(text_as_int[0:10])\n",
    "print(text[0:10])"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1115394,)\n",
      "1115394\n",
      "[18 47 56 57 58  1 15 47 58 47]\n",
      "First Citi\n"
     ]
    }
   ],
   "execution_count": 8
  },
  {
   "cell_type": "code",
   "source": [
    "1115394//101"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2025-03-10T07:02:51.302806Z",
     "start_time": "2025-03-10T07:02:51.294604Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11043"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 9
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 把莎士比亚文集分成一个一个的样本"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-10T07:02:51.314740Z",
     "start_time": "2025-03-10T07:02:51.306788Z"
    }
   },
   "source": [
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "class CharDataset(Dataset):\n",
    "    #text_as_int是字符的id列表，seq_length是每个样本的长度\n",
    "    def __init__(self, text_as_int, seq_length):\n",
    "        self.sub_len = seq_length + 1 #一个样本的长度\n",
    "        self.text_as_int = text_as_int\n",
    "        self.num_seq = len(text_as_int) // self.sub_len #样本的个数\n",
    "        \n",
    "    def __getitem__(self, index):#index是样本的索引，返回的是一个样本，比如第一个，就是0-100的字符,总计101个字符\n",
    "        return self.text_as_int[index * self.sub_len: (index + 1) * self.sub_len]\n",
    "    \n",
    "    def __len__(self): #返回样本的个数\n",
    "        return self.num_seq\n",
    "\n",
    "#batch是一个列表，列表中的每一个元素是一个样本，有101个字符，前100个是输入，后100个是输出\n",
    "def collat_fct(batch):\n",
    "    src_list = [] #输入\n",
    "    trg_list = [] #输出\n",
    "    for part in batch:\n",
    "        src_list.append(part[:-1]) #输入\n",
    "        trg_list.append(part[1:]) #输出\n",
    "        \n",
    "    src_list = np.array(src_list) #把列表转换为ndarray\n",
    "    trg_list = np.array(trg_list) #把列表转换为ndarray\n",
    "    return torch.Tensor(src_list).to(dtype=torch.int64), torch.Tensor(trg_list).to(dtype=torch.int64) #返回的是一个元组，元组中的每一个元素是一个torch.Tensor\n",
    "        \n",
    "#每个样本的长度是101，也就是100个字符+1个结束符\n",
    "train_ds = CharDataset(text_as_int, 100)\n",
    "train_dl = DataLoader(train_ds, batch_size=64, shuffle=True, collate_fn=collat_fct)"
   ],
   "outputs": [],
   "execution_count": 10
  },
  {
   "cell_type": "code",
   "source": [
    "for datas, labels in train_dl:\n",
    "    print(datas.shape)\n",
    "    print(labels.shape)\n",
    "    break"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2025-03-10T07:02:51.360575Z",
     "start_time": "2025-03-10T07:02:51.315734Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([64, 100])\n",
      "torch.Size([64, 100])\n"
     ]
    }
   ],
   "execution_count": 11
  },
  {
   "cell_type": "markdown",
   "source": [],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 定义模型"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-10T07:02:51.400498Z",
     "start_time": "2025-03-10T07:02:51.361569Z"
    }
   },
   "source": [
    "class CharRNN(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_dim=256, hidden_dim=1024):\n",
    "        super(CharRNN, self).__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
    "        #batch_first=True,输入的数据格式是(batch_size, seq_len, embedding_dim)\n",
    "        self.rnn = nn.RNN(embedding_dim, hidden_dim, batch_first=True)\n",
    "        self.fc = nn.Linear(hidden_dim, vocab_size)\n",
    "        \n",
    "    def forward(self, x, hidden=None):\n",
    "        x = self.embedding(x) #(batch_size, seq_len) -> (batch_size, seq_len, embedding_dim) (64, 100, 256)\n",
    "        #这里和02的差异是没有只拿最后一个输出，而是把所有的输出都拿出来了\n",
    "        #(batch_size, seq_len, embedding_dim)->(batch_size, seq_len, hidden_dim)(64, 100, 1024)\n",
    "        output, hidden = self.rnn(x, hidden)\n",
    "        x = self.fc(output) #[bs, seq_len, hidden_dim]--->[bs, seq_len, vocab_size] (64, 100,65)\n",
    "        return x, hidden #x的shape是(batch_size, seq_len, vocab_size)\n",
    "    \n",
    "    \n",
    "vocab_size = len(vocab)\n",
    "\n",
    "print(\"{:=^80}\".format(\" 一层单向 RNN \"))       \n",
    "for key, value in CharRNN(vocab_size).named_parameters():\n",
    "    print(f\"{key:^40}paramerters num: {np.prod(value.shape)}\")"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=================================== 一层单向 RNN ===================================\n",
      "            embedding.weight            paramerters num: 16640\n",
      "            rnn.weight_ih_l0            paramerters num: 262144\n",
      "            rnn.weight_hh_l0            paramerters num: 1048576\n",
      "             rnn.bias_ih_l0             paramerters num: 1024\n",
      "             rnn.bias_hh_l0             paramerters num: 1024\n",
      "               fc.weight                paramerters num: 66560\n",
      "                fc.bias                 paramerters num: 65\n"
     ]
    }
   ],
   "execution_count": 12
  },
  {
   "cell_type": "code",
   "source": [
    "print(65*256)\n",
    "print(256*1024)\n",
    "print(1024*1024)\n",
    "1024*65"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2025-03-10T07:02:51.408135Z",
     "start_time": "2025-03-10T07:02:51.401491Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16640\n",
      "262144\n",
      "1048576\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "66560"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 13
  },
  {
   "cell_type": "code",
   "source": [
    "sample_inputs = torch.randint(0, vocab_size, (3, 100))\n",
    "print(sample_inputs.shape)\n",
    "model = CharRNN(vocab_size)\n",
    "output=model(sample_inputs)\n",
    "output[0].shape"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2025-03-10T07:02:51.513598Z",
     "start_time": "2025-03-10T07:02:51.410119Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([3, 100])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "torch.Size([3, 100, 65])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 14
  },
  {
   "cell_type": "code",
   "source": [
    "output[0].reshape(-1, vocab_size).shape"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2025-03-10T07:02:51.524508Z",
     "start_time": "2025-03-10T07:02:51.515590Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([300, 65])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 15
  },
  {
   "cell_type": "code",
   "source": [
    "\n",
    "1024*65"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2025-03-10T07:02:51.531955Z",
     "start_time": "2025-03-10T07:02:51.525502Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "66560"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 16
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-10T07:02:51.544545Z",
     "start_time": "2025-03-10T07:02:51.533941Z"
    }
   },
   "source": [
    "class SaveCheckpointsCallback:\n",
    "    def __init__(self, save_dir, save_step=5000, save_best_only=True):\n",
    "        \"\"\"\n",
    "        Save checkpoints each save_epoch epoch. \n",
    "        We save checkpoint by epoch in this implementation.\n",
    "        Usually, training scripts with pytorch evaluating model and save checkpoint by step.\n",
    "\n",
    "        Args:\n",
    "            save_dir (str): dir to save checkpoint\n",
    "            save_epoch (int, optional): the frequency to save checkpoint. Defaults to 1.\n",
    "            save_best_only (bool, optional): If True, only save the best model or save each model at every epoch.\n",
    "        \"\"\"\n",
    "        self.save_dir = save_dir\n",
    "        self.save_step = save_step\n",
    "        self.save_best_only = save_best_only\n",
    "        self.best_metrics = -1\n",
    "        \n",
    "        # mkdir\n",
    "        if not os.path.exists(self.save_dir):\n",
    "            os.mkdir(self.save_dir)\n",
    "        \n",
    "    def __call__(self, step, state_dict, metric=None):\n",
    "        if step % self.save_step > 0:\n",
    "            return\n",
    "        \n",
    "        if self.save_best_only:\n",
    "            assert metric is not None\n",
    "            if metric >= self.best_metrics:\n",
    "                # save checkpoints\n",
    "                torch.save(state_dict, os.path.join(self.save_dir, \"best.ckpt\"))\n",
    "                # update best metrics\n",
    "                self.best_metrics = metric\n",
    "        else:\n",
    "            torch.save(state_dict, os.path.join(self.save_dir, f\"{step}.ckpt\"))\n",
    "\n"
   ],
   "outputs": [],
   "execution_count": 17
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-10T07:02:54.343757Z",
     "start_time": "2025-03-10T07:02:51.546538Z"
    }
   },
   "source": [
    "# 训练\n",
    "def training(\n",
    "    model, \n",
    "    train_loader, \n",
    "    epoch, \n",
    "    loss_fct, \n",
    "    optimizer, \n",
    "    save_ckpt_callback=None,\n",
    "    stateful=False      # 想用stateful，batch里的数据就必须连续，不能打乱\n",
    "    ):\n",
    "    record_dict = {\n",
    "        \"train\": [],\n",
    "    }\n",
    "    \n",
    "    global_step = 0\n",
    "    model.train()\n",
    "    hidden = None\n",
    "    with tqdm(total=epoch * len(train_loader)) as pbar:\n",
    "        for epoch_id in range(epoch):\n",
    "            # training\n",
    "            for datas, labels in train_loader:\n",
    "                datas = datas.to(device)\n",
    "                labels = labels.to(device)\n",
    "                # 梯度清空\n",
    "                optimizer.zero_grad()\n",
    "                # 模型前向计算,如果数据集打乱了，stateful=False，hidden就要清空\n",
    "                # 如果数据集没有打乱，stateful=True，hidden就不需要清空\n",
    "                logits, hidden = model(datas, hidden=hidden if stateful else None)\n",
    "                # 计算损失,交叉熵损失第一个参数要是二阶张量，第二个参数要是一阶张量，所以要reshape\n",
    "                loss = loss_fct(logits.reshape(-1, vocab_size), labels.reshape(-1))\n",
    "                # 梯度回传\n",
    "                loss.backward()\n",
    "                # 调整优化器，包括学习率的变动等\n",
    "                optimizer.step()\n",
    " \n",
    "                loss = loss.cpu().item()\n",
    "                # record\n",
    "                \n",
    "                record_dict[\"train\"].append({\n",
    "                    \"loss\": loss, \"step\": global_step\n",
    "                })\n",
    "   \n",
    "                # 保存模型权重 save model checkpoint\n",
    "                if save_ckpt_callback is not None:\n",
    "                    save_ckpt_callback(global_step, model.state_dict(), metric=-loss)\n",
    "                # udate step\n",
    "                global_step += 1\n",
    "                pbar.update(1)\n",
    "                pbar.set_postfix({\"epoch\": epoch_id})\n",
    "        \n",
    "    return record_dict\n",
    "        \n",
    "\n",
    "epoch = 100\n",
    "\n",
    "model = CharRNN(vocab_size=vocab_size)\n",
    "\n",
    "# 1. 定义损失函数 采用交叉熵损失 \n",
    "loss_fct = nn.CrossEntropyLoss()\n",
    "# 2. 定义优化器 采用 adam\n",
    "# Optimizers specified in the torch.optim package\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
    "\n",
    "\n",
    "# save best\n",
    "if not os.path.exists(\"checkpoints\"):\n",
    "    os.makedirs(\"checkpoints\")\n",
    "save_ckpt_callback = SaveCheckpointsCallback(\"checkpoints/text_generation\", save_step=1000, save_best_only=True)\n",
    "\n",
    "\n",
    "model = model.to(device)\n"
   ],
   "outputs": [],
   "execution_count": 18
  },
  {
   "cell_type": "code",
   "source": [
    "record = training(\n",
    "    model,\n",
    "    train_dl,\n",
    "    epoch,\n",
    "    loss_fct,\n",
    "    optimizer,\n",
    "    save_ckpt_callback=save_ckpt_callback,\n",
    "    )"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2025-03-10T07:14:41.759718Z",
     "start_time": "2025-03-10T07:02:54.344748Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "  0%|          | 0/17300 [00:00<?, ?it/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "05362b59d46d46fe9aadd5047d61c2d3"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "execution_count": 19
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-10T07:14:42.241814Z",
     "start_time": "2025-03-10T07:14:41.760712Z"
    }
   },
   "source": [
    "plt.plot([i[\"step\"] for i in record[\"train\"][::50]], [i[\"loss\"] for i in record[\"train\"][::50]], label=\"train\")\n",
    "plt.grid()\n",
    "plt.show()"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ],
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAikAAAGdCAYAAADXIOPgAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjEsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvc2/+5QAAAAlwSFlzAAAPYQAAD2EBqD+naQAAWYpJREFUeJzt3XdYVFf+BvD3zjAMoAxFpBdR7AJ2RWOLPaa4SUzWFNPbmk0xa7Lml6LJJroxWTcbs266m2RdExM1m2g0WLA3FFQsKFgApUkb6jDMnN8fM3NhpChF5+K8n+fxGbhz78yZr8Pwcu4550pCCAEiIiIihVE5ugFEREREDWFIISIiIkViSCEiIiJFYkghIiIiRWJIISIiIkViSCEiIiJFYkghIiIiRWJIISIiIkVycXQDrobZbMbFixfh6ekJSZIc3RwiIiK6CkIIlJaWIjg4GCpV8/tF2kVIuXjxIsLCwhzdDCIiImqBzMxMhIaGNvu4dhFSPD09AVhepE6na7PHNRqN+O233zBp0iRoNJo2e9z2hnVgDQDWwIZ1YA0A1sCmtXXQ6/UICwuTf483V7sIKbZTPDqdrs1DioeHB3Q6ndO/CZ29DqwBa2DDOrAGAGtg01Z1aOlQDQ6cJSIiIkViSCEiIiJFYkghIiIiRWJIISIiIkViSCEiIiJFYkghIiIiRWJIISIiIkViSCEiIiJFYkghIiIiRWJIISIiIkViSCEiIiJFYkghIiIiRXLqkPLV7vP48awKqTmljm4KERERXcapQ8r6lBxsz1Ehs6jS0U0hIiKiyzh1SFFZLx1tFsLBLSEiIqLLOXVIkay3ZmYUIiIixXHukGJNKYI9KURERIrj5CHFklKYUYiIiJTHqUOKytaT4thmEBERUQOcOqTUjklhTCEiIlIapw4pKp7uISIiUiynDim1Y1KYUoiIiJTGyUOK5ZZTkImIiJTHqUNK7cBZphQiIiKlceqQIsG24qyDG0JERET1OHdIkRdzc2w7iIiIqD6GFHDgLBERkRK1KqQsWrQIkiThhRdeaHK/VatWoVevXnBzc0N0dDTWr1/fmqdtM/IUZAe3g4iIiOprcUg5cOAAPvnkE8TExDS53+7duzFz5kw89thjSEpKwvTp0zF9+nSkpKS09KnbDBdzIyIiUq4WhZSysjLcf//9+Oyzz+Dj49Pkvh9++CGmTJmCuXPnonfv3nj77bcxcOBALF26tEUNbku2dVI4cJaIiEh5XFpy0OzZszFt2jRMmDABf/nLX5rcd8+ePZgzZ47dtsmTJ2Pt2rWNHmMwGGAwGOTv9Xo9AMBoNMJoNLakyQ2SrCd6TDWmNn3c9sb22lkD1qDurbNiHVgDgDWwaW0dWlu/ZoeUlStX4tChQzhw4MBV7Z+Tk4OAgAC7bQEBAcjJyWn0mIULF2LBggX1tv/222/w8PBoXoObkJerAqDC8RMnsL74eJs9bnsVHx/v6CY4HGvAGtiwDqwBwBrYtLQOFRUVrXreZoWUzMxMPP/884iPj4ebm1urnrgp8+bNs+t90ev1CAsLw6RJk6DT6drseX4tSQYK89CzVy/cMjKyzR63vTEajYiPj8fEiROh0Wgc3RyHYA1YAxvWgTUAWAOb1tbBdiakpZoVUg4ePIi8vDwMHDhQ3mYymbB9+3YsXboUBoMBarXa7pjAwEDk5ubabcvNzUVgYGCjz6PVaqHVautt12g0bfpmUassQ3Iklcqp34Q2bV3f9og1YA1sWAfWAGANbFpah9bWrlkDZ8ePH4+jR48iOTlZ/jd48GDcf//9SE5OrhdQACAuLg6bN2+22xYfH4+4uLhWNbwtcDE3IiIi5WpWT4qnpyf69etnt61Dhw7o1KmTvH3WrFkICQnBwoULAQDPP/88xowZgw8++ADTpk3DypUrkZiYiE8//bSNXkLLcTE3IiIi5WrzFWczMjKQnZ0tfz9ixAisWLECn376KWJjY/HDDz9g7dq19cKOI6g4BZmIiEixWjQFua6EhIQmvweAGTNmYMaMGa19qjbHqyATEREpl1Nfu8d2vsdsdnA7iIiIqB6nDim2nhQiIiJSHqcOKRJsY1J4uoeIiEhpnDqkqDgFmYiISLGcOqTYpiCzJ4WIiEh5nDykWFIKIwoREZHyOHVIUXExNyIiIsVy6pBSO3DWwQ0hIiKiepw6pHDgLBERkXI5dUixjZzl6R4iIiLlceqQUrssPhERESmNU4cU24KznIJMRESkPE4dUlTy6R4HN4SIiIjqceqQwsXciIiIlMvJQwp7UoiIiJTKqUMKB84SEREpl1OHFF4FmYiISLmcOqRwMTciIiLlcuqQAl67h4iISLGcOqSoeBVkIiIixXLykGK55ZgUIiIi5XHqkMKrIBMRESmXc4cUDpwlIiJSLCcPKbwKMhERkVI5dUjhYm5ERETK5dQhhVdBJiIiUi7nDim8dg8REZFiOXVIUVlfPcekEBERKY9ThxROQSYiIlIupw4pvHYPERGRcjl1SLGNSeHAWSIiIuVx8pBiuWVEISIiUh7nDinWWw6cJSIiUh6nDikqTkEmIiJSLCcPKZZbjkkhIiJSHqcOKZA4BZmIiEipnDqk2HpSiIiISHmcOqTULubGrhQiIiKlceqQwsXciIiIlMupQ4rEgbNERESK1ayQsmzZMsTExECn00Gn0yEuLg6//vpro/svX74ckiTZ/XNzc2t1o9uKfBVkB7eDiIiI6nNpzs6hoaFYtGgRunfvDiEE/v3vf+OOO+5AUlIS+vbt2+AxOp0Oqamp8ve2YKAEtad7GFOIiIiUplkh5bbbbrP7/p133sGyZcuwd+/eRkOKJEkIDAxseQuvIV4FmYiISLmaFVLqMplMWLVqFcrLyxEXF9fofmVlZYiIiIDZbMbAgQPx7rvvNhpobAwGAwwGg/y9Xq8HABiNRhiNxpY2uR6z2WS9Nbfp47Y3ttfOGrAGdW+dFevAGgCsgU1r69Da+kmimec6jh49iri4OFRVVaFjx45YsWIFbrnllgb33bNnD06fPo2YmBiUlJTg/fffx/bt23Hs2DGEhoY2+hzz58/HggUL6m1fsWIFPDw8mtPcJiXmS/gmTY0eXmbM7mNus8clIiIioKKiAvfddx9KSkqg0+mafXyzQ0p1dTUyMjJQUlKCH374AZ9//jm2bduGPn36XPFYo9GI3r17Y+bMmXj77bcb3a+hnpSwsDBcunSpRS+yMWuTsjB39XEM6+KNbx8b2maP294YjUbEx8dj4sSJ0Gg0jm6OQ7AGrIEN68AaAKyBTWvroNfr4efn1+KQ0uzTPa6uroiKigIADBo0CAcOHMCHH36ITz755IrHajQaDBgwAGlpaU3up9VqodVqGzy+Ld8sLmq15QtJcuo3oU1b17c9Yg1YAxvWgTUAWAObltahtbVr9TopZrPZrtejKSaTCUePHkVQUFBrn7ZN8CrIREREytWsnpR58+Zh6tSpCA8PR2lpKVasWIGEhARs3LgRADBr1iyEhIRg4cKFAIC33noLw4cPR1RUFIqLi7F48WKcP38ejz/+eNu/khbgYm5ERETK1ayQkpeXh1mzZiE7OxteXl6IiYnBxo0bMXHiRABARkYGVKrazpmioiI88cQTyMnJgY+PDwYNGoTdu3df1fiV60FiTwoREZFiNSukfPHFF03en5CQYPf9kiVLsGTJkmY36nqRF3NzbDOIiIioAc597R5eBZmIiEixnDqk8CrIREREyuXUIQW8dg8REZFiOXVIUfEqyERERIrl5CHFcssxKURERMrj1CHFNgXZzMv2EBERKY6ThxTLLftRiIiIlMe5Qwpsi7kxphARESmNU4cUTkEmIiJSLqcOKbx2DxERkXI5dUjhFGQiIiLlcuqQInExNyIiIsVy7pAiX7vHwQ0hIiKiepw6pHDgLBERkXI5dUiRF3NjSiEiIlIcJw8plltGFCIiIuVx7pBiveXAWSIiIuVx6pAiT0FmRiEiIlIcpw4pXMyNiIhIuZw6pLAnhYiISLmcOqRw4CwREZFyOXdIAacgExERKZVThxQu5kZERKRcTh1SOHCWiIhIuZw8pEhX3omIiIgcwrlDivWWPSlERETK49QhRSXxKshERERK5dwhxfrq2ZFCRESkPE4dUmxTkHntHiIiIuVx7pDCxdyIiIgUiyEFHDhLRESkRE4dUnjtHiIiIuVy6pDCKchERETK5dwhhT0pREREiuXUIUXFgbNERESK5dQhRZJ4FWQiIiKlcuqQwqsgExERKZdTh5S6lxfkgm5ERETK4twhpc5VkJlRiIiIlMXJQ0rt1xyXQkREpCxOHVJUdVIKr4RMRESkLM0KKcuWLUNMTAx0Oh10Oh3i4uLw66+/NnnMqlWr0KtXL7i5uSE6Ohrr169vVYPbkqpOT4rgRGQiIiJFaVZICQ0NxaJFi3Dw4EEkJibi5ptvxh133IFjx441uP/u3bsxc+ZMPPbYY0hKSsL06dMxffp0pKSktEnjW49jUoiIiJSqWSHltttuwy233ILu3bujR48eeOedd9CxY0fs3bu3wf0//PBDTJkyBXPnzkXv3r3x9ttvY+DAgVi6dGmbNL617HpSGFKIiIgUxaWlB5pMJqxatQrl5eWIi4trcJ89e/Zgzpw5dtsmT56MtWvXNvnYBoMBBoNB/l6v1wMAjEYjjEZjS5tcT01NTe1zVlfDRWpxOdo1W03bsrbtDWvAGtiwDqwBwBrYtLYOra1fs38rHz16FHFxcaiqqkLHjh2xZs0a9OnTp8F9c3JyEBAQYLctICAAOTk5TT7HwoULsWDBgnrbf/vtN3h4eDS3yY2qNgG2Emz87Te4qdvsodul+Ph4RzfB4VgD1sCGdWANANbApqV1qKioaNXzNjuk9OzZE8nJySgpKcEPP/yAhx56CNu2bWs0qLTEvHnz7Hpg9Ho9wsLCMGnSJOh0ujZ7nrLKKmD/dgDAxIkT4emmabPHbk+MRiPi4+MxceJEaDSsAWvgvDUAWAeANQBYA5vW1sF2JqSlmh1SXF1dERUVBQAYNGgQDhw4gA8//BCffPJJvX0DAwORm5trty03NxeBgYFNPodWq4VWq623XaPRtOmbxbXGLH+tVrftY7dHbV3f9og1YA1sWAfWAGANbFpah9bWrtXrpJjNZrvxI3XFxcVh8+bNdtvi4+MbHcNyvXEKMhERkXI1qydl3rx5mDp1KsLDw1FaWooVK1YgISEBGzduBADMmjULISEhWLhwIQDg+eefx5gxY/DBBx9g2rRpWLlyJRITE/Hpp5+2/StpAYmLuRERESlWs0JKXl4eZs2ahezsbHh5eSEmJgYbN27ExIkTAQAZGRlQqWo7Z0aMGIEVK1bgtddew6uvvoru3btj7dq16NevX9u+ihayn4LMlEJERKQkzQopX3zxRZP3JyQk1Ns2Y8YMzJgxo1mNul7Yk0JERKRcTn3tHgCQrGNROCaFiIhIWZw+pNjwbA8REZGyOH1IsZ3xMTOlEBERKQpDivWWGYWIiEhZGFKst+xJISIiUhaGFGtKYUYhIiJSFoYU6y1DChERkbIwpFhvOQWZiIhIWZw+pECe3ePYZhAREZE9pw8ptgJwWXwiIiJlcfqQUju7x6HNICIioss4fUiBPLuHKYWIiEhJnD6k1A6cJSIiIiVhSLHecjE3IiIiZWFI4WJuREREisSQYr1lTwoREZGyMKRYb5lRiIiIlIUhhad7iIiIFMnpQ4oNT/cQEREpi9OHFJWtJ8WxzSAiIqLLOH1IsWFPChERkbI4fUjhwFkiIiJlYkix3nJZfCIiImVhSOGYFCIiIkViSLHemnkZZCIiIkVhSLGmFGYUIiIiZWFIsd4KnvAhIiJSFKcPKTYcN0tERKQsTh9SuCw+ERGRMjGkWG+5mBsREZGyMKRYbxlRiIiIlIUhRZ7dw5hCRESkJAwp1luuOEtERKQsDCnWW2YUIiIiZWFI4WJuREREisSQYr3l6R4iIiJlcfqQYsOeFCIiImVx+pBiO93DSchERETK4vQhxVYA9qQQEREpi9OHFBuuk0JERKQszQopCxcuxJAhQ+Dp6Ql/f39Mnz4dqampTR6zfPlySJJk98/Nza1VjW5LkmQJJ8woREREytKskLJt2zbMnj0be/fuRXx8PIxGIyZNmoTy8vImj9PpdMjOzpb/nT9/vlWNbku8dg8REZEyuTRn5w0bNth9v3z5cvj7++PgwYMYPXp0o8dJkoTAwMCWtfAaqx04S0RERErSrJByuZKSEgCAr69vk/uVlZUhIiICZrMZAwcOxLvvvou+ffs2ur/BYIDBYJC/1+v1AACj0Qij0diaJtup+1hGY02bPnZ7Ynvdzvr6AdYAYA1sWAfWAGANbFpbh9bWTxItXMXMbDbj9ttvR3FxMXbu3Nnofnv27MHp06cRExODkpISvP/++9i+fTuOHTuG0NDQBo+ZP38+FixYUG/7ihUr4OHh0ZLmNmrZcRVOlqjwQJQJQzrzlA8REVFbqaiowH333YeSkhLodLpmH9/ikPLMM8/g119/xc6dOxsNGw0xGo3o3bs3Zs6cibfffrvBfRrqSQkLC8OlS5da9CKbastd/9iME8Uq/PXOvrhzQEibPXZ7YjQaER8fj4kTJ0Kj0Ti6OQ7BGrAGNqwDawCwBjatrYNer4efn1+LQ0qLTvc8++yz+OWXX7B9+/ZmBRQA0Gg0GDBgANLS0hrdR6vVQqvVNnjstXqzSCq1U78RgWtb3/aCNWANbFgH1gBgDWxaWofW1q5Zs3uEEHj22WexZs0abNmyBZGRkc1+QpPJhKNHjyIoKKjZx14LXHCWiIhImZrVkzJ79mysWLECP/30Ezw9PZGTkwMA8PLygru7OwBg1qxZCAkJwcKFCwEAb731FoYPH46oqCgUFxdj8eLFOH/+PB5//PE2fiktU3sVZKYUIiIiJWlWSFm2bBkAYOzYsXbbv/rqKzz88MMAgIyMDKhUtR00RUVFeOKJJ5CTkwMfHx8MGjQIu3fvRp8+fVrX8jYiXwXZoa0gIiKiyzUrpFzNGNuEhAS775csWYIlS5Y0q1HXExdzIyIiUianv3aP7XQPMwoREZGyOH1IsWnhTGwiIiK6Rpw+pNgKYGZGISIiUhSnDym1p3uYUoiIiJTE6UOKDXtSiIiIlMXpQwqnIBMRESkTQwpP9xARESkSQ4r1lhmFiIhIWRhSrLdczI2IiEhZGFLka/c4th1ERERkjyHFeis4dJaIiEhRnD6k2PBsDxERkbI4fUhRcXYPERGRIjl9SLHhmBQiIiJlcfqQwqsgExERKRNDivWWU5CJiIiUhSHFessxKURERMrCkGI73ePYZhAREdFlGFKstzzdQ0REpCwMKdZbZhQiIiJlcfqQAi6LT0REpEhOH1K4LD4REZEyMaRYb3m6h4iISFkYUmyne3i+h4iISFEYUqy3jChERETKwpBiveUUZCIiImVhSOG1e4iIiBSJIcV6y2XxiYiIlIUhxXrLiEJERKQsTh9SahdzY0whIiJSEqcPKZK1D4UzkImIiJTF6UOKigNniYiIFMnpQ4oNB84SEREpi9OHFC6LT0REpExOH1Jsp3uMZrNjG0JERER2nD6kaNWW2wqDybENISIiIjtOH1LcrCGl1GB0bEOIiIjIjtOHFHdbSKmqcWxDiIiIyI7ThxQ3F8uI2TKGFCIiIkVhSLH2pOgZUoiIiBTF6UOK7XRPGcekEBERKUqzQsrChQsxZMgQeHp6wt/fH9OnT0dqauoVj1u1ahV69eoFNzc3REdHY/369S1ucFuz9aRUGc0wmjgNmYiISCmaFVK2bduG2bNnY+/evYiPj4fRaMSkSZNQXl7e6DG7d+/GzJkz8dhjjyEpKQnTp0/H9OnTkZKS0urGtwVbSAE4LoWIiEhJXJqz84YNG+y+X758Ofz9/XHw4EGMHj26wWM+/PBDTJkyBXPnzgUAvP3224iPj8fSpUvxr3/9q4XNbjtqFeCuUaHSaEZpVQ18Org6uklERESEZoaUy5WUlAAAfH19G91nz549mDNnjt22yZMnY+3atY0eYzAYYDAY5O/1ej0AwGg0wmhsu7EjtsfqoHVBpbEahWWVCNJp2uzx2wtbHdqytu0Na8Aa2LAOrAHAGti0tg6trV+LQ4rZbMYLL7yAkSNHol+/fo3ul5OTg4CAALttAQEByMnJafSYhQsXYsGCBfW2//bbb/Dw8GhpkxulqjEAkLB5206c92rzh2834uPjHd0Eh2MNWAMb1oE1AFgDm5bWoaKiolXP2+KQMnv2bKSkpGDnzp2takBD5s2bZ9f7otfrERYWhkmTJkGn07XZ8xiNRsTHxyPAV4e8i6Xo238wxvf2b7PHby9sdZg4cSI0GufrSQJYA4A1sGEdWAOANbBpbR1sZ0JaqkUh5dlnn8Uvv/yC7du3IzQ0tMl9AwMDkZuba7ctNzcXgYGBjR6j1Wqh1WrrbddoNNfkzeLpbnnMSpNw6jfjtapve8IasAY2rANrALAGNi2tQ2tr16zZPUIIPPvss1izZg22bNmCyMjIKx4TFxeHzZs3222Lj49HXFxc81p6DXlqLVmNS+MTEREpR7N6UmbPno0VK1bgp59+gqenpzyuxMvLC+7u7gCAWbNmISQkBAsXLgQAPP/88xgzZgw++OADTJs2DStXrkRiYiI+/fTTNn4pLdfRjSGFiIhIaZrVk7Js2TKUlJRg7NixCAoKkv9999138j4ZGRnIzs6Wvx8xYgRWrFiBTz/9FLGxsfjhhx+wdu3aJgfbXm/sSSEiIlKeZvWkCCGuuE9CQkK9bTNmzMCMGTOa81TXlafck+LcU82IiIiUxOmv3QMAHa09KWUG9qQQEREpBUMK6vakMKQQEREpBUMK6vSkMKQQEREpBkMKamf36DkmhYiISDEYUlA7u4djUoiIiJSDIQWAp5tlRbySSvakEBERKQVDCoBOHVwBWAbOVteYHdwaIiIiAhhSAAA6Nxe4qCQAQGF5tYNbQ0RERABDCgBApZLga+1NuVRmcHBriIiICGBIkXXqaLnqcgF7UoiIiBSBIcXKr6O1J6WUPSlERERKwJBiZRs8W1DOkEJERKQEDClW8umeMp7uISIiUgKGFCs/a0i5xJBCRESkCAwpVp068nQPERGRkjCkWNkGzvJ0DxERkTIwpFh16mAbk8KeFCIiIiVgSLGyne65VF4NIYSDW0NEREQMKVa2npTqGjNKeTVkIiIih2NIsXJ3VaODqxoAx6UQEREpAUNKHX6elt6UPH2Vg1tCREREDCl1hPq4AwAyiyod3BIiIiJiSKkj3LcDACCjsMLBLSEiIiKGlDrCfT0AABkF5Q5uCRERETGk1BHRyRJSzrMnhYiIyOEYUuqw9aRkMqQQERE5HENKHeHWnpRLZdUo41opREREDsWQUofOTQMfDw0A9qYQERE5GkPKZWynfM4XMKQQERE5EkPKZcI72aYhc4YPERGRIzGkXCaqc0cAwJGsEge3hIiIyLkxpFxmZFQnAMDOtEswmXk1ZCIiIkdhSLlMbJg3PLUuKK4wIuUCe1OIiIgchSHlMhq1CiOsvSmv/HgECal5Dm4RERGRc2JIacCo7p0BACdzSvHI8gM4d4mDaImIiK43hpQG3BIdhJ4BngAAIYCkzCIHt4iIiMj5MKQ0wLeDKza+OBoPj+gCADiapUdxRTWMJrNjG0ZEROREGFKaEB3iBQD4du95DHlnE/7841EHt4iIiMh5MKQ0ITrUElKqTWYYTQL/O3wBJRVGB7eKiIjIOTCkNKGbdWE3G6NJYOOxHAe1hoiIyLk0O6Rs374dt912G4KDgyFJEtauXdvk/gkJCZAkqd6/nBzl/7JXq6R6234+ctEBLSEiInI+zQ4p5eXliI2Nxccff9ys41JTU5GdnS3/8/f3b+5TO8Rr03pDkiy3ALA7vQBF5dUObhUREdGNz6W5B0ydOhVTp05t9hP5+/vD29u72cc52mM3ReL+YRFwd1Xjh4NZOJlTiu2n83FH/xBHN42IiOiG1uyQ0lL9+/eHwWBAv379MH/+fIwcObLRfQ0GAwwGg/y9Xq8HABiNRhiNbTdw1fZYV3pMFwkwGs0Y090PJ3NKsel4Dm7p2z56gq7G1dbhRsYasAY2rANrALAGNq2tQ2vrJwkhWnwVPUmSsGbNGkyfPr3RfVJTU5GQkIDBgwfDYDDg888/xzfffIN9+/Zh4MCBDR4zf/58LFiwoN72FStWwMPDo6XNbbV0PfCPYy7wUAv8ZYgJ6vpDVoiIiMiqoqIC9913H0pKSqDT6Zp9/DUPKQ0ZM2YMwsPD8c033zR4f0M9KWFhYbh06VKLXmRjjEYj4uPjMXHiRGg0mivuX2MyY/hfE1BSWYOvHhqEjxPS4ePhio9nxkKS2m9iaW4dbkSsAWtgwzqwBgBrYNPaOuj1evj5+bU4pFy30z11DR06FDt37mz0fq1WC61WW2+7RqO5Jm+Wq31cjQYY3zsAqw9dwNwfU3CpzBKkTuRWIDbMu83bdb1dq/q2J6wBa2DDOrAGAGtg09I6tLZ2DlknJTk5GUFBQY546lZ7dGQkAMgBBQD+d5jTkomIiNpas3tSysrKkJaWJn9/9uxZJCcnw9fXF+Hh4Zg3bx4uXLiAr7/+GgDw97//HZGRkejbty+qqqrw+eefY8uWLfjtt9/a7lVcR/1CvDCqux92nL4kb/v58EW8ekvvBtdVISIiopZpdk9KYmIiBgwYgAEDBgAA5syZgwEDBuCNN94AAGRnZyMjI0Pev7q6Gi+99BKio6MxZswYHD58GJs2bcL48ePb6CVcfy9N6gkfDw3+NKkHdG4uyCs14MC5Qkc3i4iI6IbS7J6UsWPHoqmxtsuXL7f7/uWXX8bLL7/c7IYpWf8wbyS9MQkAkJ5fjjVJF5CQmo/hXTth/9lCnMzR48HhEe16MC0REZGj8do9rTS6hx8AYMfpfBhNZjz1TSLe+OkY9qQXOLhlRERE7RtDSivdFNUZAHDsoh4/H76IIutVknelX2rqMCIiIroChpRW6uypRZ8gy9zvV348Im9nTwoREVHrMKS0gbE9Lb0pRlPtWJ0jWSUoN9Q4qklERETtHkNKG3hqdDeMjOoEAAjxdkeItztqzAJf7zmPksra6xak5ZWijMGFiIjoqjCktAEvDw2+fWwYfng6DiufHI4R3SyB5a8bTmLSkm04mlWCvWcKMOFv2zFv9VEHt5aIiKh9cMiy+DciSZIwuIsvAGD2uChUm8xIPFeEC8WVmPnZXtwUZZkFtPVkHg6eL8S6Izl4ekxX+OvcHNlsIiIixWJPyjXQxa8DPvz9AGx4YRR6BXqizFCDDcdyAABlhho8/u9EfLnrLG7+YBvO5Jc5uLVERETKxJByDXm6afDQiC71ttumKZcZavDehlS7+0qrjDCazNejeURERIrGkHKN3RoTBDdN/TK7WK/zk3i+SF7B92SOHiMWbcEDn++T99t0PBdnL5Vfn8YSEREpCEPKNebppsHvBoQAAKb2C5S3z5nUA2qVhEtlBmSXVKHcUIOnvzmI0qoa7DtbiFx9FTak5ODxrxNx17Ldjmo+ERGRw3Dg7HXw5m19Mb1/CHoF6rDpRC5qzAK3xwbjl8PZOJ6tx9wfDuPQ+WJUGk3yMXvSC/DLkWwAQGF5NWpMZriomSmJiMh5MKRcB24aNYZ1tUxL/mzWYNSYBEJ9PBAb5o3j2XrsSrOsThvm6w5fD1cczirB7vRLSK8zqDazqBKRfh0c0n4iIiJH4J/m19nYnv6Y0CcAANA/zEvePizSF9vnjsOLE3sAAL5PzLIbi5KWZwksBWUG1HBgLREROQGGFAeKDfOWv54zsQckScKQLr7yoNq60vLKkHiuEEPf3YwXvku+fo0kIiJyEIYUB+oZ4InHb4rECxO6y6eDOmhdMGNwmLyPX0dXAJaQsnz3OZjMAr8cyeYFDImI6IbHMSkOJEkSXru1T73tC++MxsuTeyJHX4X0/DI8uyIJSRlFyCqulPd5+cfDGBDmgz1nChDq4477hoZjxuAwVBlNeHZFEnoHeeKlST2v58shIiJqUwwpCuXTwRU+HVyhkiynfs5Yx6d09euAoopqZBZWIrPQElrySw04klWCiX0CsPdMATadyMWmE7mI69oJI6zL8RMREbU3DCkK18XPA2qVBJPZsuDbY6MiMb5XAH47noM8vQFx3TrhjZ9SkJ5fjq2pedh/tkg+dv7Px7D8kaFYvDEVo3v44XcDQh31MoiIiJqNIUXhtC5qvDSpBw6eK8JtscG4o38wJEnCrLgu8j5T+wVh6dY0xB/PxeHMEnn7qdwyjFi0BQCw7kg2JvcNhIcr/8uJiKh94G+sduAPY6OavH9inwAs3ZqG9UctFzF0VavwyYOD8If/HJIXiKs2mbE26SKCvN1wIluPib0D0D3A85q3nYiIqKUYUm4A0SFeCNS5IUdfBQAYEumDcb388Z8nhmHFvgwYasz4+fBFvLrmqHzMyv2Z+O3F0XDTqB3VbCIioiZxCvINQKWS8M7v+mFYpC8i/Trg8Zu6AgAGhvvg/RmxeH1ab6ita6/4dnCFt4cGGYUV+GLnWQBAjr4KG7Mk/JqS47DXQEREdDn2pNwgxvcOwPjeAQ3e569zw9KZA3C2oBwPDI/A5hO5ePG7w1i6JQ1Gkxkfb02D0aTGxlVHMbxbZ/jr3K7qOSuqa6BWSdC6sDeGiIjaHkOKk5gaHSR/fUdsCFYfuoAdpy/h75tOy9tNZoF5q4+i0mjC8+NrF5hrSEGZAbcv3YUyQw3euzsGk/sGYm3SBRy7WIIAnRseHRkJVQMr5xIR0dVJyytFdY1An2Bdo/tUGU2oMprg7eF6HVt2/TCkOCGVSsLS+wbirmW7kZZXht8PCYVUeB7/TVdj88k8AEBOSRXG9fLHqdxSfHBPLPw6aPFBfCr0lTWYf3tfvLPuBC5YF5d76puDePO2Pljw83H5OXoGemJU984OeX1ERO3duUvlmPC37XDXqLFn3s0NhhCzWeDBL/Yh5YIePz07Ev9LvojoUC9M7htot58QAkaTgKtL+xvhwZDipLzcNVg7eyRSc/SIDuqIn345h1VnJdRY12M5c6kcZ6xjVp74dyJiw7zx9Z7zAABPNxesTroASQKGdPHF/rOFeGfdCbvHP5JVApPZcrXnKP+O1/fFERG1Y0II/N9ay0SHSqMJSRnFGNfLHyWVRrioJHTQWn51r0m6gAPnLGtjPbr8ALKKKiFJwF/vjME9Q8JQYzJj47Fc/GPzaZy5VIaVTw7HwvUnoVGr8NF9A+DXUeuw13i12l+sojbTUeuCQRG+kCQJrmrgr3f2w50DQjB7XDd5Hw9XNQ5nlcgBBQD+mZAOAJg5NBxvWJf1t4Wb2FDLlZ1/PJiFh786gPs+24sqownp+WUQwrJPak4p3v7lOMoMNdfldRIRKcXp3FLc99lefHcgQ95WZTThnwlp2HvGck22+OO52JVWe322pIwiJGUU4aa/bsHUD3egxmRGldGExRtT5X2yiiw920IAf159BNtP5ePmD7Zh9opDSM0thdEk8MZPx5B4vgh7zhTg7mW7cSq3FKPe24KHvtyPGpP5OlWgediTQrLbY4Nw1+BwGGpMKKk0okeAJ/qHeeMfm0+joLwafYN1+Hav5QdLo5bw7LgoBHm5oVegJ07mlMLDVY3Z46Lw5DcH5WX880oNuH3pTpzKLcPCO6Px+yFhmPz37QAAnZsGz0/o7rDXS0TUlMoa4NW1x3D3oDB5jF5plREdtS6QpJaNuXvrl+PYnV6A3ekFyCysxEMjumDmZ3uRlleGIC837P7zzfj3nnMALL3WpVU1+PHQBXy1+xxKq2pQWlWD/ecKkZpTihx9FUK83WEyC+Toq+DtoUHfYB12pRXgia8TYagxo5P1EitpeWU4dlEvt+NcQQXuWrYbpVU1yCysxD8T0vHceOV9HrMnherRuqjxl+nRmBXXBTGh3vj8oSFY84eR+Mv0aESHWHpK7h4UimBvd0iShPuHhQMApkUHYWCET73HO5VbBgD4Z0IaDmUUy9vPF5Rf+xdDRNRC359RYdXBC7j3070AgK0n8xA9/zd8sv1Msx6noMwAk1ngaFYJdpy+BNucgo8T0vDE14lIy7N8RmaXVGFrah52pRVAJQFL7ukPALhQXInSqtqe51+OZOOTbZY2zB4XhUdGdgEAPDOmm7z4p6HG0jPy5cNDsPLJ4XbtuXdwGADYPeaHm08jObO4Wa/remBPCjXLwjujsSoxE89P6CFve2B4BLp27oj+Yd7ooHVBgE6LXL0BAOCiqh3nkllYiTd+SpGP01cZAVh+gD/ZfgY7Tl/C8+O7Y0o/+0FfNmazwH/2ncegCN8mR7sTEbWFlCL73pJVBzMBAD8czMLTY7o1dIido1kleG/jSew4fQkPDo/ApTLL5+Id/UNQUV2DjcdykZxZDJVk+eOw0mjC/62xfEaO7x2Acb387R7v/27pjXfWn8CKfZYe7UCdG+4aFAJXtQqT+gaiSycPAEB3/444nVeGcT07IzbMGwDQM8ATqbmlAIDnJnSHocaEtckXMaJbJ/h11CoyoAAMKdRM/UK80M/am2IjSRJG1rnacp8gHXL1+Yj064AXJnRHak4pjl6w/AVRt7sxs7ASQgg8/nUikqw9LPP/dww39/JvcBT66qQLeP2nY+jauQM2zxnT4u5WIqLLHc4sxqc7zmDe1F44nVeG7KIKVJtrP2Oqa8zYefoSACAtrww5JVUI9Kq/ptSpXMuYu6KKahy7qId1KB6+3Vc7ru+Zsd1gtA5qBYAZg8Lg7aHBJ9vPILvEsnL4rLgIqFUSYkK9cCSrBFP6BuLBuAi8s752ksKfp/aS16mK9Osgb397ej98uv0MXreOGQSAuG6dkJpbiki/Dgjxdsfb0/uhT7AO0/uHwM1VDZUkoaNWeZGAp3uozQ3u4gsAuLmXP+7oH4KXp/TCvUPC5PvvHmS5GnNmUQW2n76EpIxieUXcHH0V1iZfaPBxfzyYBQA4k1+O03llMJsFjl/Uo7LadC1fDhG1QkV1Df664STS88uu23MWlldj3ZFsmK29uA0pqTRiWUI6isqrUVltwh0f78K6I9lYuP4knvw6Ea+uPWa3/2/Hc6Cvc3pkZ9qleo95OLMYdy/bjR2nLyHlgiWg3B4bjHBfDwhhGdQ6sU8AegR4om+wFx67KRJ9gnSYM6kHhlg/NwGga+cOGNnN8off4rtj8eKEHvjgnli4adR4ZGQXeLq54J/3D8T0ASENvrbhXTvhy4eH2AWXuweFwstdgweGRwAAPN00eHJ0N/jr3KBz0ygyoADsSaFr4LGbIhHm64GJdVbAnRYdhJp7Bbp27oAeAZ744WAWKqpNeOtnywfBwyO6wN9Ti4W/nsS/tqVjdPfOWHkgAzOHhiNA54asogrsOVM72n3h+hPI0RtwIluPewaH4r27Y6/76ySiK/vv/kwsS0jHqZxSfPHwkOvynP+35ih+TcnBm7f1wSMjI5FZWIHzBRW4qXttj+/fN53CV7vOIaOwAp061K5Bsu5odoOPaTvFYvPehpPoHWQJG0aTGS4qCfNWH4W+qgYDw73x1JhuCPF2R78QL2w7lY+HvtwPAPjD2NrTRHV7OrRdavsMHhweIS+G2TPQEz0Day8G++ZtffHatD7yH3ZXq1+IFw6/OalZxygBQwq1OTeNGrfHBtttkyTJLvXbxq2k55dDo5bw5Oiu8HBVY9m2dJzJL8fwhZsBAJfKDPjL9Gj8lHwRAKB1UcFQY8bW1Hz5sX5NycGiO2OQnFWMP65IwstTeuKO/g3/hUFEV89kFliffAEjo/xavKbGaes4iIMZRRBCXPVp2uKKani5a664//mCcuSXGuQe3DJDjbwo5feJWTCazFi8MRVGk8BnswZjYp8ACCGw6YTlVMuWk7l2A0gbszvd8kfS9P7BWJt8EXmlBkz7x068Nq03liWkw0OrRmZhJTq4qvH5Q0PgWyf4jO7uh+dujoKbqxoDwutPLgAAbw9XzBwahtScUrm3uTHNDSjtGU/3kEOE+XjIX4/o5ocAnRs83TSYbR2ZbrP+qOWih5utHygvTOgBF+sP6M3WQWWlVTVIzS3FkvhTuFBciW/qrOmSX2rArC/341/b0q/p6yG6Ea1OuojnVybj3fUnmtyvtMrY6DobttM8xRVGnCuouKrn3X+2EAPfjscz3x6C2Sxw7GIJhr6zCasSM+32qzKaMONfe3D3v/Yg8VwhAGDLyTxUW2e2nMjW4931J2E0WU77/HbM8nmSlleGzELLuiK5egMqqk0I83VHqI+7/NgLbuuNOf1qcNdA+z+4XpzYAzMGhSLCOkj1L+tOoKC8Wn68R0ZG2gUUwPJH2pxJPeWZN41ZeGcMVv9hJDzdNFdVJ2fAkEIOEeZbG1LqLuH8YFwEQrxrPyhKq4zILzXII89v7x+MpfcNxF+m98PnswZjdA/L0vvfHcjEDuugtiNZJagymlBdY8YDn+/D9lP5WPTrSRhqTDCZBZbvOovj1gG8VUYT/rH5NE7nXr/z5UTtxcEMy2qmh5uY+XEqtxTD3t2Ml1YdbvD+M/m1Sw0kZRThYnEl/vzjEaTmlNbbd/OJXHx/IBOf7zgDswA2HMvBPxPS8PPhbOSVGuT1Q2zWJF1AXqllxsz7v6Vi3ZFs/Hv3uXqPOzDcGwCw7VQ+hBByT0tdt8YEY2hk7biQUd07IcITdp9HId7uiOjUAYtnxGL9c6MQZB046+OhwVjrTJrHR0U2XChqEYYUcgiNura7ckKf2ml2bho1/vP4MHzx0GD4dnCF0STw1a6zMAugq3VU+pR+gXjAes52eFfLh8ryOh9M1SYzDmcW47MdZ+QpdwBw8HwRVh/Kwvyfj+PlHy0fqJ9tP4O/xZ/CH787DNsYu+KKatz8fgKe+fagXZszCytwNKukrUtBdF1kFVVg9n8O4djFq38PH79o+fk5V1ABQ03DA9Tf/OkYKqpN+Cn5IsxmASEE9p8tREmlEcUV1Sgor5b3TcooxkdbTmPlgUxM/vt2VFTXnmbJKqrAU98cxMs/HsFvx3Pl7R9uPo0ka1g6dlGP9zacxMxP96KgzIDPdtSuV7L3TCFmrziEg+ct+/7fLb0hScCE3v5Y8cRwuGvUyCs14ER2KdZbx53YpuwCwK0xQRhmDSnBXm4ItYaT0DohZVidENNB64IP7olFnyAdPvz9ACx/ZCh+mj3yhr3Qn6MwpJBD9A+znJdVqyT4e9pP4+vi1wHjewfIC8fZluEfVWfQm83wy67UbPur58C5Qqw+lGV3347Tl+SZQ8cu6lFcUY3vrN3H6fnlSC6wBKfPdpzBmUvl+DUlR17LpcZkxj2f7MHtH+/E7gZG9RMp3SfbzmDd0Wx8tDmt3n0r9mXgvQ0nYaozG6bGDJy2LjJmMgucta4iXVJhxDd7z+NQRhHKDTVIyiySj8korMCO05dwzyd78Oqao0jPt1+wMTmzWO7xBID3NtQu6/7p9jPymkoAMDjCByHe7jCaBPadtZzKEcLyebDnTAGe+uYgzuSXw9PNRR4D19lTi2nRQfjz1F54fFQk9s0bj08fHAw3jRpx3SyfFfP/dwxHskqgdVFh4Z0x0Kgl9AvRoU+QDrfGBOOO/sF4dVpveSxMiE/t59Plnzcjuvlh/fOj5B5danscOEsOce+QMJiEwNgmfrhjQy2j4m1uauCqyjEhXhgZ1QlF5Ua8OLEHsooqsODn4/h6z3nklRrg6qLC67f2wetrU/DDwSx5MSUhgH9sTpOvdwEA36apsOPDXfKS/gDw/YFMbDuVj2GRvvL6BXN/OIKNL46uN2Vv+a6zKK824YHhEfByb/ycclZRBVYfuoBHb4ps02l/VUYTvt5zDpP7BiKiU4crH0AO992BDJwvqMDcyT2v+bo/u9It4SAp034Aq77KiNd/SoHJLBAb5i3PysuugF1o+OfWdOSXGnDwfBGqTWb4dXTFS5N6ospYOxbl2EU9jmdbemqOZpXgjHU8StfOHXAmvxzHLpag7qTgb/eex+OjIpFTUoWVByx/MMSGeuFwVgkeHxWJn5Ivyldbv1yitcfkgeEReHFCD9wzOAwDI7zh4Vr7M+Wvqw0Yk/sGYMvJPOy3jl15YlRXxHXrhA0vjIaPhyskyXLhvg9/PwAAYDRa/kCpe7pnWNfanhS6PhhSyCHUKgkPWufrNyYm1Fv+um+wrsGeFBe1Cv95vHbJZ9tYE9t56nE9O2NK30C8vjYF+dZtNl/uslzleXr/YOw9U4AcvcEuoACWQXEA7P76sw3OVauAymoznhsfhRPZpZj/83EAllNP6/54k90HpI3ZLDB2cQJqzAISgD+O7w6zWcBQY4a7q7rJelzJq2uOYvWhC9h0PA/fPx3Xqseia6/KaMJra1NgNAncEh1Ub5HE5jCZBQw1Jni4uuDzHWeQVVSJyX0DYTSZMSjCB/oqozw2JFdvwLGLeqgkCb2DPLHvTKHcg/LOuhP406rDuH9oGIrL7UPT/w5ftPv+Ulk1lm6x75U5nl0iL9iYVVSBk9ZxJzdF+aFTB1f5ir2Rfh0Q7O2GXWkFuOmvWyFJlj8chkb6YuUTw5FtvSbNuYIK/JqSU+/12vbXqCU8PKILXF1UdtOLG3L3oDBkFVVi6dY0BHi64akxXQEA3To3fZX2YC83/H5IGDRqFcLrjKWj66PZp3u2b9+O2267DcHBwZAkCWvXrr3iMQkJCRg4cCC0Wi2ioqKwfPnyFjSVnM1N3f0woXcAnhgViR+fGQE3zZV/ifcJ1uHxm2oHrt0WG4zOnlrcVmdKdN2w46KS8OzN3RH/wk14tX8NnhoVCR8Pjd0iSHXZupW/3Xse764/iSWbTuF0Xhk2pNSurZBfasDG47kQQuDlHw5j/AcJKLSel1+bfEH+69Q2pXHhryfQb/5G+bz7O+uOY/i7m5Fy4erHDuirjFh9yHIqa/+5QvmK0y2Vnl/W6F+wSmA0mXEkq7jVr/N6SMsrwyNf7ccvR+x/yadcKJFnnTS10FlJpRGHMooavE8IgTnfJaPna7+i75sbsfDXE/jLuhNYvvscZn62F7O+3I+3fj5ud0VdALj1o5245R87cO8ne/Hf/bXrf2QUVqC0qgb/2n4WmdaQ4lmnty/E2x2//PEmjO1p6dW0vUceHtEFgOWPBFtIMQtga6plgGpXvw64Z3Dtgo5Du/jimTG1M12EAGYMCsVnswZDpZLk3ouY0Nrg1jPAEzo3F0T6dcCjIy0/43cNDEVAA38MNEStkvDSpJ7YPnccfnnupqueQSNJEhbdFYO3p/fjKtcO0OyQUl5ejtjYWHz88cdXtf/Zs2cxbdo0jBs3DsnJyXjhhRfw+OOPY+PGjc1uLDkXN40anz80GP83rc9VBRSb127tg3/ePxDP3RyFKdaZQx/NHICk1ydi+9xx+Mv0fvK+j94UiSj/jnDTqBHgDvxpUnckvTEJC27vW+9xg73c8MrUXgBg9wt8V9olbLBObbR9uKZklWD90Rx8n5iF9Pxy/JqSDZNZ4G/xp+TjckurUFxRja/3nIfJLPBT8kVkFlbgy13nkKOvwuwVh+QxMYBlLEBj/nvZQlNnLzV88cai8mrctWx3k1Oys0uqMP6Dbbhj6a4mV+y8GvmlBnlqaFv6aPNp3L50F1YdzGp0nyqjCYcyilBlbP2KxMUV1Sg3XHktjbpO5ZbiTH4Z3vxfCram5uPZFUl2C4LZBngCQHpewyGl3FCDaf/YgTv/uRsHz1tO08xbfQQ3f5CAPekF2JNegNVJluArBOSLztW16UQudlnHUblctr7G/nOF2GKd6VJ3+i0AHLpk2ffW2CB52yMju6BfiJe8GioARHTywK0xln0STuXb9Vjaem96B+lwS3QQOlh7C4dG+mJkVCeM69kZOjcXfDRzABbPiK13mjQ6xAu2XDA00hdb/jQWa2ePxCtTeuHj+wZifgM/p1cS5uvR4jVf6Ppr9umeqVOnYurUqVe9/7/+9S9ERkbigw8+AAD07t0bO3fuxJIlSzB58uTmPj3RVbklOgi3RAfZbfOxXrJcCIGJfQJQUGbA841cmjy6Ttf7fcPCMSzSF939PRHi7Y6+wTq7axB9s/c8zuSXw0Ul4fkJ3fHyD0ew72wBtqTWTnNMSM1HmI+H3RiYc5fK8Z99GfLVSvdYe1ZsXe/nCyow/3/H8MGMWCz4+TiW7z6HxXfHYFKfQGQWVSDM10P+UP/+svUjDmUUI0dfhef+m4T5t/fFrTGWHqC/bzqFg+eLcPB8UaMXSIs/YWn3pTIDLhRX2k0XtzGbBSQJTf5laVnDYjfOFVTgx2fiMCii7c7n2655sivtkt1f6Dbxx3Pxyo9HUFhejSdHd8Wrt/RGldGEdUeyMaVfIDo0YyzQpTIDJvxtGyJ8PbB29khUGk1Y8L/jGBjhjXuHhMv7mcwCJrOAq4sKeaVVuH3pTtSYhN24jnfWHceMwaHQqFV2vSPp+eWorDZh1cFMuKpVuHdIGPLLDHhj7TH5PZN4rhAXiyvx3/2W/+sHvtgnv1dmDg3DvrOFOJNfDkkC4l8cg3BfD8Qs2IiC8mq5F+eugaHyYPHZ47rh4621YfWHp0cgq6gCSzadwq60AlSaJLhpVPjD2Ch8n5gFk1lghrXWtkGoADCupz96B+nkUzCXc1FJiA3zhptGjTdv74ttp/IxNToQkiTh84csK9A2tjiZp5sG3Tp3RFpeGXoEdLQLF9Nigho8hm4s13xMyp49ezBhwgS7bZMnT8YLL7zQ6DEGgwEGQ20a1+stvxCMRqM8mKkt2B6rLR+zPXLGOvxzpm0ZfWH3vrLddnSV0KWTB84VVGBCTz/5FJHRaMS4Hn44dlEvfyjb/loc0c0Xw7t4A0C9Rat2p12Cyjpk8P6hYfj1WA4Ky41YvLF2dkNqbmntVUpv7oaPtqZj9aELuFRahe2nLQHmuwMZ+OfWNJwtqIBaJeHTBwYgyMtNXrn3zgEh+C4xC4nnCnCxuBKXyqrx+Y4zmNzb0j1f9xdjlaHa7peD7bUn1AlXJy4Ww9NVwh/+m4zoYC/8aVJ3ZBRWYPqyvRjfszMW3x3daI3/lXBGrsMvhy8iJtiz0X0bIoTApzvO4Zcj2YAk4auHBsKvoxYF5dVynY5fLJHbXWMyw0WtgtFkxqurj8in2BJO5mHuxCh8GH8ay7afxcnsErw8ufYq3gajCUs2p8FoEnh1ak+YTTV29dh6IgfFFUYUV5TgXH4pvkvMwneJmfjxUBYGhXkhopMHjCYz7ly2F2XVJqx5ejjWJF2wG1B67+AQbDyWh+JKIw5nFCImRGfXk7Ir7RLGvb8VOdarh28+kYuEU/ny6SDAssbIp9stPSVd/TrI46c0agnPjI7EtH4BePTrQ7g9JggRPlpAmDA4wgc70wpgNAkE6rSYPTYSPx+5iBBvN/xhdCRiQnR45j9JGNGtEzp5qNHJwxNxkb7y6aGx3f0Q6KnBD08Og7urGh4ulrp093OHj4cGRRVGjO3RCa4qgcl9ArDhWO3UYZs+wZ5Qwwyj0YzfxQbid7GBsP3c2Zib6Oz6w5hI/HDoAib17nxdP6Oc8XOxIa2tQ2vrJ4lWnNSVJAlr1qzB9OnTG92nR48eeOSRRzBv3jx52/r16zFt2jRUVFTA3d293jHz58/HggUL6m1fsWIFPDw4cImuj8wyILtCwpDOAnU7DAqqgKXH1RjsJ/DbhdozpnNjahDiAbyWqEZZjeWAaWEm7MhRQW+sfYCXomvwc4YKp0osx2rVAl4aIK/Ksk+XjgIv9DNh9TkVtuc0fUa2h5cZUTqB9Zlq9PE2Y7i/wJen1NBpBMpqALOQIEHgL4NNcFcDfz6glq/sumBgDbwv6/U2mIB5B9QwCcs+t4Wb0FED/DddLT/OtmyV/Lqf61uDrp7AjhwJO3NV8HYVeLq3GeU1wIJDahitzxXoLjCvf/3fREIAAoAtK23PtnwxOkgguUDCV6dqT/NNCzNhUqj9dhUE3htmwoZMFbZkS3i2jwllRglf1jlOgsBfh5rwYYoaFyokRHQUmBNtwoF8CavPqaBVAUXVlued2c2ETlog2EOgg/XMw3/SVNifb3m9U0LNiL8gyfWJ8TXjoe5mHCmU8O/TluccE2jGab2EixUSdBqBYA+B+6LM+P6MCilFKtwRYUKsr8BbSfX/RvRQC1SYat8rXToKBHkI7MlTQQUBMyT4uAq8NsCE9FIJu3Ml9PISiAsQ8v+fRlVbz00XJPycYWnXhBAzbgs3Q18NuKoAN+vTF1QBHTWA1lqy86XA31Isdz7W04QY34Z/RZwqkZBTAYwKtPx8VNQA8w5YjgvtIJBlHdMyJsiMO7s0vBot3fgqKipw3333oaSkBDqdrtnHK3J2z7x58zBnzhz5e71ej7CwMEyaNKlFL7IxRqMR8fHxmDhxIjQa512GmHVofg0evNNy++c1Kfjx0EXMm9IDj47sAgD48PROlFl7EF65dxzctqThx0OW7vY+QZ54asZw5K9Pxam9lvEJT42OQm5pFb5LtAx8XfrQCPQM9MQYQw1e/jEFLioJDwwPwx9XHkZhueWvkkHh3jiUWYxTJSqcL1cBMOOBsf0wrmdn/Hvxdujr/PEiIEET3h9Bvu6o3ndA3l7i2wtbzhfj9Vt7IcLXA0ajEe+v3CT/AgYAlU8oLlRUAyiAgARzcAxSz5wDYHl9u8v8EN4rFD/uTQEA5FZK6NxnKPIu6mE0n0KYjzsuFFcip1JC/xHjEFxnOueJ7FLM/fEoqmvMWPuH4cgrNeD5v+8CANwxbhC2/3ISQDk8XNWoqDbhWIUnlkwdiQPrTgKwnLIwQ0Jp537YvP8khADSpVCUmI0ACvDUqEj8dPgicvQGdOg2CBf2Wxbwu1ipwrgJE7DoH7tQUVOFCtTOFll5Rg0hgI4agcV3x2J87wAs+mAHAMv08w1ZlrDSK9ATqbmlOFKowpvJriirM15lR64KZuvMky1zx8mn5Mp3nkPKxlMocw+EKtQPSDqB/mFeOHZRL/eYLLo7FsmZJfjh0AW8OCEK9w8Nw7mCCkz6cBfMsPy/TIoJw+3WC9O92OS7FAjNKsHPn+wDAPzprlHo1vnKU9NrTGasy9+DEn0Z/njXzejg3vD4jVsa2Db0pgr8mpKL6BAdHlpuWQzxrtH9MbVfYAN7Kxs/Fy1aWwfbmZCWuuYhJTAwELm59l2Aubm50Ol0DfaiAIBWq4VWW/8HQ6PRXJM3y7V63PaGdWh+Dd6eHo1Hb+qKvsG1Y1iGd/PD2YIM6NxcEO7niZcm9YKXuxZd/DwwtV8QXF1d0bfOmJcnx0bhQlEldpwuwGOjuqJfmGXsho9Gg88eqr1q7IhufpZTHwAeH9UVK61ruBhqzNC6qDAlOgS+HVwxb2oveeq0X0ctLpUZsOrQhXrjR/62yTJ9NH35Qex85WYAQJreuoCVtyVcHDhXJE/nBoD340+jpNIIN40KQgCJ54uRVVRl97gbjudhv3XxrafGdMOapAs4eL4ICzechrurGhXVNQjQuWHl/kxUW6/3siO9CBfqjNeZveIwSg018PbQYMPzozHu/QScK6hA8oUy7LSeinBVq1BtMmPBLyfl4+JP5MmPed/wCJwvrMSGYzn4bGft9ZyMJoFlO87J6968Pb0fBkf44NHlB+RtZUYJs1cewSMjI+VtdS2+OxbJWcX4aPNpu/rEde0kX6371phg+Olqe37jojoDG0/hYEYxqq2hZHLfICRn1s7imhwdjNsHhOH12/rK/1/dAjRwdVHJ16MZEeV31e/RARGd8PCILtC5a9Ar2PuqjtFogJ9nj8C69b+ig7u2WT8P3QK88GyAF4orqqGSLDN8hnXr3K4/V/i5aNHSOrS2dtc8pMTFxWH9+vV22+Lj4xEXx3UcqP3zcHWxCygA8MqUnujgqsaDcZZ1YIK93fHGbX3s9pk+IASncsswsU8AOmpd0DPQE7vnjW/yuUZGWUKK1kWFMT07w91Vje2n8+HXUYsPZsTKFzV77KZIVFSbsP1UPp69OQoPf3VAXp/CTaOCr4crLtb5xZtVVIljF0sQ5qXFmVLLL8b7h4fjvQ2p8n62q1aXVFq6aMb3CoBvB1d8s/c8cvSWff4yvR9eW5uCb609RFoXFW6LDUZltQkHzxfJM6Aasv5oNvL0tb/sSw01kCTg1am9Eejlhqn9ArE66QKe+fYgCsqroXNzwaS+gfjBOrvHr6MrtC5qedbVnQNCENGpA2LCvLDhWI587SebZdZVjGeP6yav17PsgUFYlZiJeweF4J1Vu7AvX4UvdlrW0ukV6Cmv+dE3WIfoUC9Eh3rh/qHhWLE/A//YfBp3DwrFc+O7IynDMjX68qvd9g3WwV2jRnGFUV53Z2KfAKw6mIkz+eWIDvGC1sVyzqVuoFSrJER17ojj2Za/SC9f9bQpKpXUohkwkiShNRfa9fZwxV/vioEArnqKMFFDmh1SysrKkJZWu4DP2bNnkZycDF9fX4SHh2PevHm4cOECvv76awDA008/jaVLl+Lll1/Go48+ii1btuD777/HunXr2u5VECmIt4crXru1T5P7aF3UeP0K+1xuar9ArErMxNie/vBwdcHYnv6If3E0Ar3c7VaulSQJz43vjuesM5eW3BuL347lQqWSMGdiD3yfmFlvquq0f+yEl7sLSiotv5l+NyDEbsny58Z3x/qj2diVVoBIvw54cnRX+Hi44j/7zsMsgCFdfHDvkDD8dcNJ+bL3d/QPhpe7Bo/eFIkwXw/sTr8ETzcXuKrVSMsvwx2xwQjQueG2pTvlq10DwJ+n9sKutEt4dlwUhll/If9xfHdsPpknXwfmT5N7QuuikkPK4hmxOHiuCEu3psFT64I/32KZKt6/zoKAgGXdjSPW6y+5aVR4oM6Cgv3DvNE/zBtGoxEzu5kxqE8UViddhMbFUrdPtp/BoYwiu/83lUrCA8Mj7B6n7syXujRqFe4eFIpv9lp6dfw9tYjy74ilMwdi+e6z+NOkng0eBwA9Az1xPFuPrn4d2s0v/RkNzLoiaq5mh5TExESMGzdO/t42duShhx7C8uXLkZ2djYyM2rUAIiMjsW7dOrz44ov48MMPERoais8//5zTj4maydvDFav/MNJuW5T/lWfM/G5AKH43IFT+PtjL/jSrrVu+pNISLkK93RDk5Y6RUZ2wK60AT43pivuGhuPewWEoraqBT53L0P9uQCh+PJSFmUPDoVGr8Pz47vhq1zncNSgUz1inOKtVEqb0C8SUBsYlCCEQ7uuBjELLGJfoEC88PaZbvenRkX4d8MmDg/DwV/vRO0iH+4dFQF9pxPeJWRjf2x/jevojOsQLp3JLcfegUPl6UAMjfDAowgeF5dWY2CcAcd064ZGvLONy/n5vfwR5NXzKWZKAFydE4eWpveVtAyN8kF9qQO+glo+Le/3WPjhzqQy70grw+yGWX+J9gnV47+7YJo8b0sUXa5Iu4OZe/k3uR3SjaXZIGTt2bJOrPDa0muzYsWORlJTU3KciomvAdnl5wHI9k/m390X88Vy88dMxAJbTEgDwj98PwPnCCgwI84YkSXBRS3YBBQDevbMfHhoRIa8r8/iornh8VNerboskSfjjzVH4+6bT6BXoiWdvjmp03+FdO2HfvAlwc1VBrbK05cdnRsj3+3XU4tNZg+2OcdOo7fYxmQWeGmMZQzSlX/PW2fDrqG31ImCuLip88dAQ7E6/hBHdml7Gva57h4QhzNcdQ7rw2jHkXBQ5u4eIrp26M2x6BeoQ5OWOmUPD5ZAyuItlLEWnjlp0usIvZa2L2u4aSy0xY3DYVZ8a8PJo3SA8tUrCvDq9I47gplHj5l4BzTpGrZIwqoELbBLd6Jq9LD4RtW91Q0rvIMvpIo1ahTVPD8fEEDNmDg5t7FAiouuKIYXIyfh4aOS1O+rOTOoXosOt4WZom3GdJCKia4mne4icjCRJ+OTBQSgoq27wujxERErBkELkhJqz1gYRkaPwdA8REREpEkMKERERKRJDChERESkSQwoREREpEkMKERERKRJDChERESkSQwoREREpEkMKERERKRJDChERESkSQwoREREpEkMKERERKRJDChERESkSQwoREREpUru4CrIQAgCg1+vb9HGNRiMqKiqg1+uh0Wja9LHbE9aBNQBYAxvWgTUAWAOb1tbB9nvb9nu8udpFSCktLQUAhIWFObglRERE1FylpaXw8vJq9nGSaGm8uY7MZjMuXrwIT09PSJLUZo+r1+sRFhaGzMxM6HS6Nnvc9oZ1YA0A1sCGdWANANbAprV1EEKgtLQUwcHBUKmaP8KkXfSkqFQqhIaGXrPH1+l0Tv0mtGEdWAOANbBhHVgDgDWwaU0dWtKDYsOBs0RERKRIDClERESkSE4dUrRaLd58801otVpHN8WhWAfWAGANbFgH1gBgDWwcXYd2MXCWiIiInI9T96QQERGRcjGkEBERkSIxpBAREZEiMaQQERGRIjl1SPn444/RpUsXuLm5YdiwYdi/f7+jm9QiCxcuxJAhQ+Dp6Ql/f39Mnz4dqampdvuMHTsWkiTZ/Xv66aft9snIyMC0adPg4eEBf39/zJ07FzU1NXb7JCQkYODAgdBqtYiKisLy5cuv9cu7avPnz6/3Gnv16iXfX1VVhdmzZ6NTp07o2LEj7rrrLuTm5to9RnuvQZcuXerVQJIkzJ49G8CN+T7Yvn07brvtNgQHB0OSJKxdu9bufiEE3njjDQQFBcHd3R0TJkzA6dOn7fYpLCzE/fffD51OB29vbzz22GMoKyuz2+fIkSMYNWoU3NzcEBYWhvfee69eW1atWoVevXrBzc0N0dHRWL9+fZu/3sY0VQej0YhXXnkF0dHR6NChA4KDgzFr1ixcvHjR7jEaev8sWrTIbh8l1+FK74WHH3643uubMmWK3T7t/b1wpRo09PkgSRIWL14s76Oo94FwUitXrhSurq7iyy+/FMeOHRNPPPGE8Pb2Frm5uY5uWrNNnjxZfPXVVyIlJUUkJyeLW265RYSHh4uysjJ5nzFjxognnnhCZGdny/9KSkrk+2tqakS/fv3EhAkTRFJSkli/fr3w8/MT8+bNk/c5c+aM8PDwEHPmzBHHjx8XH330kVCr1WLDhg3X9fU25s033xR9+/a1e435+fny/U8//bQICwsTmzdvFomJiWL48OFixIgR8v03Qg3y8vLsXn98fLwAILZu3SqEuDHfB+vXrxf/93//J1avXi0AiDVr1tjdv2jRIuHl5SXWrl0rDh8+LG6//XYRGRkpKisr5X2mTJkiYmNjxd69e8WOHTtEVFSUmDlzpnx/SUmJCAgIEPfff79ISUkR//3vf4W7u7v45JNP5H127dol1Gq1eO+998Tx48fFa6+9JjQajTh69Og1r4EQTdehuLhYTJgwQXz33Xfi5MmTYs+ePWLo0KFi0KBBdo8REREh3nrrLbv3R93PEaXX4UrvhYceekhMmTLF7vUVFhba7dPe3wtXqkHd156dnS2+/PJLIUmSSE9Pl/dR0vvAaUPK0KFDxezZs+XvTSaTCA4OFgsXLnRgq9pGXl6eACC2bdsmbxszZox4/vnnGz1m/fr1QqVSiZycHHnbsmXLhE6nEwaDQQghxMsvvyz69u1rd9y9994rJk+e3LYvoIXefPNNERsb2+B9xcXFQqPRiFWrVsnbTpw4IQCIPXv2CCFujBpc7vnnnxfdunUTZrNZCHHjvw8u/1A2m80iMDBQLF68WN5WXFwstFqt+O9//yuEEOL48eMCgDhw4IC8z6+//iokSRIXLlwQQgjxz3/+U/j4+Mg1EEKIV155RfTs2VP+/p577hHTpk2za8+wYcPEU0891aav8Wo09Mvpcvv37xcAxPnz5+VtERERYsmSJY0e057q0FhIueOOOxo95kZ7L1zN++COO+4QN998s902Jb0PnPJ0T3V1NQ4ePIgJEybI21QqFSZMmIA9e/Y4sGVto6SkBADg6+trt/0///kP/Pz80K9fP8ybNw8VFRXyfXv27EF0dDQCAgLkbZMnT4Zer8exY8fkferWzLaPkmp2+vRpBAcHo2vXrrj//vuRkZEBADh48CCMRqNd+3v16oXw8HC5/TdKDWyqq6vx7bff4tFHH7W7MKczvA9szp49i5ycHLv2enl5YdiwYXb/797e3hg8eLC8z4QJE6BSqbBv3z55n9GjR8PV1VXeZ/LkyUhNTUVRUZG8T3upC2D5nJAkCd7e3nbbFy1ahE6dOmHAgAFYvHix3am+G6EOCQkJ8Pf3R8+ePfHMM8+goKBAvs/Z3gu5ublYt24dHnvssXr3KeV90C4uMNjWLl26BJPJZPdBDAABAQE4efKkg1rVNsxmM1544QWMHDkS/fr1k7ffd999iIiIQHBwMI4cOYJXXnkFqampWL16NQAgJyenwXrY7mtqH71ej8rKSri7u1/Ll3ZFw4YNw/Lly9GzZ09kZ2djwYIFGDVqFFJSUpCTkwNXV9d6H8gBAQFXfH22+5raRyk1qGvt2rUoLi7Gww8/LG9zhvdBXbY2N9Teuq/H39/f7n4XFxf4+vra7RMZGVnvMWz3+fj4NFoX22MoSVVVFV555RXMnDnT7qJxzz33HAYOHAhfX1/s3r0b8+bNQ3Z2Nv72t78BaP91mDJlCu68805ERkYiPT0dr776KqZOnYo9e/ZArVY73Xvh3//+Nzw9PXHnnXfabVfS+8ApQ8qNbPbs2UhJScHOnTvttj/55JPy19HR0QgKCsL48eORnp6Obt26Xe9mXhNTp06Vv46JicGwYcMQERGB77//XlG/OK+XL774AlOnTkVwcLC8zRneB9Q0o9GIe+65B0IILFu2zO6+OXPmyF/HxMTA1dUVTz31FBYuXHhDLA//+9//Xv46OjoaMTEx6NatGxISEjB+/HgHtswxvvzyS9x///1wc3Oz266k94FTnu7x8/ODWq2uN7MjNzcXgYGBDmpV6z377LP45ZdfsHXrVoSGhja577BhwwAAaWlpAIDAwMAG62G7r6l9dDqdIkOAt7c3evTogbS0NAQGBqK6uhrFxcV2+9T9P7+RanD+/Hls2rQJjz/+eJP73ejvA1ubm/pZDwwMRF5ent39NTU1KCwsbJP3hpI+U2wB5fz584iPj7frRWnIsGHDUFNTg3PnzgG4cepg07VrV/j5+dm9/53lvbBjxw6kpqZe8TMCcOz7wClDiqurKwYNGoTNmzfL28xmMzZv3oy4uDgHtqxlhBB49tlnsWbNGmzZsqVeN1xDkpOTAQBBQUEAgLi4OBw9etTuB9T2IdanTx95n7o1s+2j1JqVlZUhPT0dQUFBGDRoEDQajV37U1NTkZGRIbf/RqrBV199BX9/f0ybNq3J/W7090FkZCQCAwPt2qvX67Fv3z67//fi4mIcPHhQ3mfLli0wm81yiIuLi8P27dthNBrlfeLj49GzZ0/4+PjI+yi5LraAcvr0aWzatAmdOnW64jHJyclQqVTyKZAboQ51ZWVloaCgwO797wzvBcDS0zpo0CDExsZecV+Hvg+aNcz2BrJy5Uqh1WrF8uXLxfHjx8WTTz4pvL297WY1tBfPPPOM8PLyEgkJCXZTxioqKoQQQqSlpYm33npLJCYmirNnz4qffvpJdO3aVYwePVp+DNvU00mTJonk5GSxYcMG0blz5wanns6dO1ecOHFCfPzxx4qafvvSSy+JhIQEcfbsWbFr1y4xYcIE4efnJ/Ly8oQQlinI4eHhYsuWLSIxMVHExcWJuLg4+fgboQZCWGaqhYeHi1deecVu+436PigtLRVJSUkiKSlJABB/+9vfRFJSkjxrZdGiRcLb21v89NNP4siRI+KOO+5ocArygAEDxL59+8TOnTtF9+7d7aadFhcXi4CAAPHggw+KlJQUsXLlSuHh4VFvyqWLi4t4//33xYkTJ8Sbb755XacgN1WH6upqcfvtt4vQ0FCRnJxs9zlhm6Gxe/dusWTJEpGcnCzS09PFt99+Kzp37ixmzZrVburQVA1KS0vFn/70J7Fnzx5x9uxZsWnTJjFw4EDRvXt3UVVVJT9Ge38vXOnnQQjLFGIPDw+xbNmyescr7X3gtCFFCCE++ugjER4eLlxdXcXQoUPF3r17Hd2kFgHQ4L+vvvpKCCFERkaGGD16tPD19RVarVZERUWJuXPn2q2PIYQQ586dE1OnThXu7u7Cz89PvPTSS8JoNNrts3XrVtG/f3/h6uoqunbtKj+HEtx7770iKChIuLq6ipCQEHHvvfeKtLQ0+f7Kykrxhz/8Qfj4+AgPDw/xu9/9TmRnZ9s9RnuvgRBCbNy4UQAQqampdttv1PfB1q1bG3z/P/TQQ0IIyzTk119/XQQEBAitVivGjx9frzYFBQVi5syZomPHjkKn04lHHnlElJaW2u1z+PBhcdNNNwmtVitCQkLEokWL6rXl+++/Fz169BCurq6ib9++Yt26ddfsdV+uqTqcPXu20c8J2xo6Bw8eFMOGDRNeXl7Czc1N9O7dW7z77rt2v8CFUHYdmqpBRUWFmDRpkujcubPQaDQiIiJCPPHEE/X+MG3v74Ur/TwIIcQnn3wi3N3dRXFxcb3jlfY+kIQQonl9L0RERETXnlOOSSEiIiLlY0ghIiIiRWJIISIiIkViSCEiIiJFYkghIiIiRWJIISIiIkViSCEiIiJFYkghIiIiRWJIISIiIkViSCEiIiJFYkghIiIiRWJIISIiIkX6f90po4c8fZLDAAAAAElFTkSuQmCC"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "execution_count": 20
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 推理"
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "\n",
    "#下面的例子是为了说明temperature\n",
    "logits = torch.tensor([400.0,600.0]) #这里是logits\n",
    "\n",
    "probs1 = F.softmax(logits, dim=-1)\n",
    "print(probs1)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2025-03-10T07:14:42.262717Z",
     "start_time": "2025-03-10T07:14:42.242804Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0., 1.])\n"
     ]
    }
   ],
   "execution_count": 21
  },
  {
   "cell_type": "code",
   "source": [
    "logits = torch.tensor([0.04,0.06])  #现在 temperature是2\n",
    "\n",
    "probs1 = F.softmax(logits, dim=-1)\n",
    "print(probs1)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2025-03-10T07:14:42.268147Z",
     "start_time": "2025-03-10T07:14:42.263714Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.4950, 0.5050])\n"
     ]
    }
   ],
   "execution_count": 22
  },
  {
   "cell_type": "code",
   "source": [
    "import torch\n",
    "\n",
    "# 创建一个概率分布，表示每个类别被选中的概率\n",
    "# 这里我们有一个简单的四个类别的概率分布\n",
    "prob_dist = torch.tensor([0.1, 0.45, 0.35, 0.1])\n",
    "\n",
    "# 使用 multinomial 进行抽样\n",
    "# num_samples 表示要抽取的样本数量\n",
    "num_samples = 5\n",
    "\n",
    "# 抽取样本，随机抽样，概率越高，抽到的概率就越高,1代表只抽取一个样本，replacement=True表示可以重复抽样\n",
    "samples_index = torch.multinomial(prob_dist, 1, replacement=True)\n",
    "\n",
    "print(\"概率分布:\", prob_dist)\n",
    "print(\"抽取的样本索引:\", samples_index)\n",
    "\n",
    "# 显示每个样本对应的概率\n",
    "print(\"每个样本对应的概率:\", prob_dist[samples_index])"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2025-03-10T07:14:42.287774Z",
     "start_time": "2025-03-10T07:14:42.269141Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "概率分布: tensor([0.1000, 0.4500, 0.3500, 0.1000])\n",
      "抽取的样本索引: tensor([2])\n",
      "每个样本对应的概率: tensor([0.3500])\n"
     ]
    }
   ],
   "execution_count": 23
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-10T07:14:44.778175Z",
     "start_time": "2025-03-10T07:14:42.288769Z"
    }
   },
   "source": [
    "def generate_text(model, start_string, max_len=1000, temperature=1.0, stream=True):\n",
    "    input_eval = torch.Tensor([char2idx[char] for char in start_string]).to(dtype=torch.int64, device=device).reshape(1, -1) #bacth_size=1, seq_len长度是多少都可以 （1,5）\n",
    "    hidden = None\n",
    "    text_generated = [] #用来保存生成的文本\n",
    "    model.eval()\n",
    "    pbar = tqdm(range(max_len)) # 进度条\n",
    "    print(start_string, end=\"\")\n",
    "    # no_grad是一个上下文管理器，用于指定在其中的代码块中不需要计算梯度。在这个区域内，不会记录梯度信息，用于在生成文本时不影响模型权重。\n",
    "    with torch.no_grad():\n",
    "        for i in pbar:#控制进度条\n",
    "            logits, hidden = model(input_eval, hidden=hidden)\n",
    "            # 温度采样，较高的温度会增加预测结果的多样性，较低的温度则更加保守。\n",
    "            #取-1的目的是只要最后，拼到原有的输入上\n",
    "            logits = logits[0, -1, :] / temperature #logits变为1维的\n",
    "            # using multinomial to sampling\n",
    "            probs = F.softmax(logits, dim=-1) #算为概率分布\n",
    "            idx = torch.multinomial(probs, 1).item() #从概率分布中抽取一个样本,取概率较大的那些\n",
    "            input_eval = torch.Tensor([idx]).to(dtype=torch.int64, device=device).reshape(1, -1) #把idx转为tensor\n",
    "            text_generated.append(idx)\n",
    "            if stream:\n",
    "                print(idx2char[idx], end=\"\", flush=True)\n",
    "    return \"\".join([idx2char[i] for i in text_generated])\n",
    "\n",
    "\n",
    "# load checkpoints\n",
    "model.load_state_dict(torch.load(\"checkpoints/text_generation/best.ckpt\", weights_only=True,map_location=\"cpu\"))\n",
    "start_string = \"All: \" #这里就是开头，什么都可以\n",
    "res = generate_text(model, start_string, max_len=1000, temperature=0.5, stream=True)"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "fca90233ae8e4af3872f58920268a48c"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All: I charge you, if you were a day to seek in this,\n",
      "Whose misery shall be the senate, or the streets,\n",
      "And then resign forth the poor worship for me.\n",
      "\n",
      "HENRY BOLINGBROKE:\n",
      "What says me, sir, the sorted sir,\n",
      "I would with the stone shall be so far off from the hollow of the senate with shriek\n",
      "Marcius shall be of the fire,\n",
      "Were spider should encounter there.\n",
      "\n",
      "NORTHUMBERLAND:\n",
      "Nay, if thou desire and take his friends will stand to death; but who comes here?\n",
      "\n",
      "SICINIUS:\n",
      "Where is he not dance.\n",
      "\n",
      "ROMEO:\n",
      "In all the first of thy native resolve you.\n",
      "\n",
      "DUKE VINCENTIO:\n",
      "The son of mine honour,\n",
      "Or else he shall find\n",
      "me your father's house,\n",
      "Who lets as a head of that consent of her hatred by me; but for Edward's grave;\n",
      "And by his horse in head\n",
      "As I have heard the stars: therefore follow me, my gracious lord, I think, if she be constant,\n",
      "The easily wonder with him.\n",
      "\n",
      "First Citizen:\n",
      "We have an honest friend.\n",
      "\n",
      "KING RICHARD III:\n",
      "So long live an apile in a villain,\n",
      "A gracious part in the commonwealth!\n",
      "Why, then is h"
     ]
    }
   ],
   "execution_count": 24
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-10T07:14:44.784627Z",
     "start_time": "2025-03-10T07:14:44.779165Z"
    }
   },
   "source": [
    "res"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"I charge you, if you were a day to seek in this,\\nWhose misery shall be the senate, or the streets,\\nAnd then resign forth the poor worship for me.\\n\\nHENRY BOLINGBROKE:\\nWhat says me, sir, the sorted sir,\\nI would with the stone shall be so far off from the hollow of the senate with shriek\\nMarcius shall be of the fire,\\nWere spider should encounter there.\\n\\nNORTHUMBERLAND:\\nNay, if thou desire and take his friends will stand to death; but who comes here?\\n\\nSICINIUS:\\nWhere is he not dance.\\n\\nROMEO:\\nIn all the first of thy native resolve you.\\n\\nDUKE VINCENTIO:\\nThe son of mine honour,\\nOr else he shall find\\nme your father's house,\\nWho lets as a head of that consent of her hatred by me; but for Edward's grave;\\nAnd by his horse in head\\nAs I have heard the stars: therefore follow me, my gracious lord, I think, if she be constant,\\nThe easily wonder with him.\\n\\nFirst Citizen:\\nWe have an honest friend.\\n\\nKING RICHARD III:\\nSo long live an apile in a villain,\\nA gracious part in the commonwealth!\\nWhy, then is h\""
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 25
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
